#!/usr/bin/python
######
#       coordinator.py
#       Copyright Etelos Inc. 2007
#       Darien Hager
#       darien@etelos.com
######
import time, os, sys

#from lockwrapper import lockwrapper
from guiadapter import guiadapter
from httpmodule import httpmodule, ReagentHTTPError
from filemodule import filemodule, folder, file
from modhandler import modhandler
from subscribemodule import subscribemodule
import cPickle, threading, opener

	
		
class coordinator:
	
	PUBCHECKTIME = 60
	SYNCMARGIN = 4
	
	PLAT_WIN = 0
	PLAT_MAC = 1
	PLAT_LINUX = 2
	
	TREE_STORE = "monitor_tree.dat"
	SUBS_STORE = "subscribed.dat"

	LOG_INFO = 0
	LOG_WARN = 1
	LOG_ERROR = 2
	
	
	
	def __init__(self,rootdir, httpmod):
		""" The coordinator requires a root directory for file storage, defined as an absolute path, and an initialized HTTPmodule instance. """
		self.__loglevel = 0
		#self.__lock = lockwrapper()
		self.__lock = threading.RLock() # Must be safe for multiple threads...										
		self.__expecting = {}	# So we can ignore file events we initiate
		self.__logtext = [] # 2 item tuples, level and message

		assert os.path.isabs(rootdir), "Invalid root directory, must be absolute path: '"+str(rootdir)+"'"
		self.__root = self.__sanitizeRoot(rootdir)						
		
		self.__hmod = httpmod
		self.__fmod = self.__loadTreeStore()					
		self.__smod = self.__loadSubscriptions()		
		self.__ga = guiadapter(self)
		self.__modhand = modhandler(self.__fmod,self)
		self.__fmod.setTarget(self.__modhand,"file_flag")
		
		
	def startComponents(self):
		""" Start the worker threads that the coordinator manages, specifically 
		the dedicated file monitor and the modification checker/handler. They can
		be stopped with stopComponents()"""
		self.__lock.acquire()

		self.__modhand.start()
		self.__fmod.start()
		
		self.__lock.release()		
	def setLogLevel(self,lev):
		""" Sets the current logging level, e.g. LOG_INFO """
		if(lev in (LOG_INFO,LOG_WARN_LOG_ERROR)):
			self.__loglevel = lev
			return True
		else:
			return False
	def stopComponents(self):
		""" Stop the worker threads. They can be restarted with startComponents() """
		self.__fmod.stopThread()
		time.sleep(1) # Give a little time to make sure final events go through
		self.__modhand.stopThread()
		time.sleep(1)
	def getGuiAdapter(self):
		""" Returns a reference to this coordinator's GUI adapter object. """
		return self.__ga
	def __sanitizeRoot(self,root):
		""" Sanitize a given directory path. """
		root = os.path.realpath(root) # Take off symlinks before normpath'ing
		root = os.path.normpath(root)
		root = os.path.normcase(root)
		
		# Ensure that the __root is trailed by a slash, which will
		# vary by OS. Mac OSX / Linux it is "/", and on Windows "\"
		
		if not root.endswith(os.sep):
			root += os.sep
			
		if(not os.path.isdir(root)):
			msg = "Root directory ("+str(root)+") is invalid"
			raise IOError, msg
		return root		
	def updateSchemas(self):
		""" Trigger the coordinator to check with the server to see if any 
		schemas (particularly binders) have changed. This should be called
		whenever you suspect that a binder (or contact, etc) has either had
		a variable added/removed or else changed in datatype. """
		#self.__smod.cleanEntries()
#		schemaHash = self.__hmod.specialSchemaSum()
#		wants = self.__smod.checkHashes(schemaHash)
#		if(len(wants) > 0):
#			self.log(self.LOG_INFO,"Subscriber needs to update resources: "+str(wants))
#		schemas = {}
#		for key in wants:
#			data = self.__hmod.listRemote((key,"*"))
#			schemas[key] = data
#		self.__smod.updateSchemas(schemas)
			
	def __loadSubscriptions(self):
		""" Loads the subscriptions from a file."""
		#schemaHash = self.__hmod.specialSchemaSum()
		#schemas = {}
		#for key in schemaHash:
		#	data = self.__hmod.listRemote((key,"*"))
		#	schemas[key] = data
		
		try:
			handle = open(self.__root+os.sep+self.SUBS_STORE,"rb")
			subdata = cPickle.load(handle)
			s = subscribemodule()
			s.setFromPicklable(subdata)		
			handle.close()			
			return(s)
		except IOError, e:
			self.__log(self.LOG_WARN,"Subscription data could not be loaded from "+self.SUBS_STORE+"...")			
		except EOFError:
			# Probably the last save failed! Log it.
			self.__log(self.LOG_WARN,"Subscriptions could not be loaded. Possibly corrupt pickle data. Starting fresh.")
	
		# Errored out, so do the default					
		s = subscribemodule()
		return s
			
	def __loadTreeStore(self):
		""" Load the last record of the root directory's contents' state 
		(e.g. modtimes, etc.) """
		
		try:
			handle = open(self.__root+os.sep+self.TREE_STORE,"rb")
			treeroot = cPickle.load(handle)			
			handle.close()
			return filemodule(self.__root,treeroot)			
		except IOError:
			self.__log(self.LOG_WARN,"File states could not be loaded from "+self.TREE_STORE+"...")
			return filemodule(self.__root,folder(''))			
		except EOFError:
			# Probably the last save failed! Log it.
			self.__log(self.LOG_WARN,"Error loading last file system state. Possibly corrupt pickle data. Starting fresh.")
			return filemodule(self.__root,folder(''))
	def getRoot(self):
		""" Get the path to the root directory this coordinator is working on. """
		# No locking for this one
		return self.__root
	def saveStates(self):
		self.__lock.acquire()
		# Clean up the subscribe module's data structures for housecleaning
		typelist = self.__smod.listContents(())
		for type in typelist:
			self.__smod.cleanEntries((type,))
		
		# Let the pickling commence!
		shandle = open(self.__root+os.sep+self.SUBS_STORE,"wb")
		cPickle.dump(self.__smod.givePicklable(),shandle)			
		shandle.close()
		
		thandle = open(self.__root+os.sep+self.TREE_STORE,"wb")
		cPickle.dump(self.__fmod.getState(),thandle)			
		thandle.close()
		self.__lock.release()
		return
		
### Adding new files
		
	def addDir(self,pathbits,recursive = False):
		""" Add a specified remote directory and the files within it to the 
		local storage, also subscribing it as necessary. Takes an optional
		parameter for recursion (default: false, no recursion). 
		
		Returns a list of pathbits added."""
		self.__lock.acquire()				
		added = []	
		pathbits = self.tuplizePathbits(pathbits)
		contents = self.__hmod.listRemote(pathbits)
				
		added.extend(self.addFileSet(pathbits))
		
		if recursive:
			for item in contents:			
				if(item == ""):
					continue;		
				if(item.endswith("/")): # Server always uses "/", whereas os.sep may vary
					item = item[:-1] # Strip slash
					# It's a subdirectory						
					newbits = list(pathbits[:])
					newbits.append(item)		
					added.extend(self.addDir(newbits,recursive))
		self.__lock.release()
		return added
		
	def addFileSet(self,pathbits):		      
		"""  If the pathbits variable specifies a single file or pseudofile, 
		it is downloaded to disk and subscribed. If you specify a folder, it will
		download and subscribe all files within that folder, but does not work
		recursively.
		
		You may also specify any DB object, and it will download all pseudofiles
		for that object. (e.g. an instance of a binder, single contact, 
		publication, or issue)
		
		Returns a list of added filenames, or empty list on failure"""
		pathbits = self.tuplizePathbits(pathbits)
		added = []
		self.__lock.acquire()
		files = []
		try:
			files = self.__hmod.getRemote(pathbits)
		except ReagentHTTPError, e:
			self.log(self.LOG_ERROR,"Unable to get file(s) from server for "+os.sep.join(pathbits)+" ...\nError: "+e.message)
			
		for file in files:			
			# See also coordinator.remoteCheck() for more time-delta stuff
			ourmodtime = file.modtime - self.__hmod.timeDelta() 
			#print("MTIMEDEBUG:\t"+"Remote:"+str(file.modtime)+" Diff:"+str( self.__hmod.timeDelta())+" Local:"+str(ourmodtime))
			newbits = list(pathbits[:])						
			if(file.name != ""):
				newbits.append(file.name)
			if(self.__writeFile(newbits,file.data,ourmodtime)):
				added.append(pathbits);
				success = self.__smod.add(tuple(newbits))
				if(success):
					self.log(self.LOG_INFO,"Subscribed: "+str(tuple(newbits)))
				else:
					self.log(self.LOG_WARN,"Subscribe failure: "+str(tuple(newbits)))
		
		self.__lock.release()
		return(added)

				
	def __writeFile(self,pathbits,data,modtime = False):
		""" Writes the given data to the local storage folder in the appropriate
		place for it's pathbits value. An optional modtime can be applied.
		
		The benefit of this function is that it manages the file monitor to
		avoid firing events for the creation or editing of a new file. (Most
		of those events are intended to represent unexpected external changes.)"""
		
		pathbits = self.tuplizePathbits(pathbits)
		filepath = self.__root + os.sep.join(pathbits)
		# Check if file exists before operation, store
		existed =os.path.exists(filepath) 
		if not existed:
			# Make sure the directories are in place to support it
			parentdir = self.__root + os.sep.join(pathbits[:-1])
			if not os.path.exists(parentdir):
				os.makedirs(parentdir)
		
		if not existed:
			self.__expecting[pathbits] = "ADD" # Ignore next creation
		else:			
			# Ignore events in the next second on this file...
			self.__expecting[pathbits] = int(time.time())+1
			
		
		# Do writing or creation as necessary
		file = open(filepath,"wb")
		file.write(data)
		file.close()
		
		# Check that our modtime can be converted to integer
		if(modtime != False):
			try:
				modtime = int(modtime)
			except ValueError, e:
				self.log(self.LOG_WARN,"Attempting to write file with modtime '"+str(modtime)+"', but is not integer.")
				modtime = False
				
		# Check valid modtime...
		if(modtime != False):
			# Set modtime on file 							
			os.utime(filepath,(modtime,modtime)) # Tweak modtime
		else:
			# Not set? Retrieve actual for later ignore-listing
			modtime = int(os.stat(filepath).st_mtime)	
		
		# Re-add the subscription, setting the ignore value just in case 
		# we completed this function so quickly that the tracker notices
		# right after this.
		if not existed:
			self.__expecting[pathbits] = "ADD" # Ignore next creation
		else:			
			self.__expecting[pathbits] = modtime # Ignore next change
				
		return True
		
### === Expectations and subscriptions		
	
	def getExpect(self,pathbits):
		""" Get expectation data for a given pathbits. If none set, returns False. 
		The expectation values are used to know which file events to ignore,
		usually because those events were triggered by the coordinator in the
		first place.
		
		There is no publicly-accessible setExpect() function."""
		pathbits = self.tuplizePathbits(pathbits)
		self.__lock.acquire()		
		if(pathbits) in self.__expecting.keys():
			ret = self.__expecting[pathbits]
		else:
			ret = False 
		self.__lock.release()
		return ret
	def delExpect(self,pathbits):
		""" Removes an expectation for a given pathbits. """
		pathbits = self.tuplizePathbits(pathbits)
		self.__lock.acquire()
		if(pathbits) in self.__expecting.keys():
			del self.__expecting[pathbits]
			ret = True
		else:
			ret = False 
		self.__lock.release()
		return ret
	def isSubbed(self,pathbits):
		""" Tests if a given resource is subscribed or not. Subscribed files are
		judged "important" by the system and modification events on them will
		trigger upload attempts, etc."""
		pathbits = self.tuplizePathbits(pathbits)
		self.__lock.acquire()		
		ret = self.__smod.exists(pathbits)
		
		self.__lock.release()
		return ret
	def listContents(self,pathbits,decorate = False):
		""" List the contents of a particular pathbits path, if possible. 
		If the pathbits designate a file or pseudofile object, then a blank array
		is returned. An empty pathbits tuple will return a list of valid top-level
		resource types.
		
		If the optional "decorate" value is True, then results in the list that 
		are analogous to folders will be suffixed with the system-dependent OS
		seperator. ("/" on *nix, "\" on Windows, etc.)
		"""		
		# Decorate means non-file items have a path separator (os.sep) appended
		pathbits = self.tuplizePathbits(pathbits)
		self.__lock.acquire()
		ret = self.__smod.listContents(pathbits,decorate)	
		
		self.__lock.release()
		return ret
	def unsub(self,pathbits):
		""" Unsubscribe a resource described by the pathbits string. Returns
		True on success, False on error."""
		pathbits = self.tuplizePathbits(pathbits)
		self.__lock.acquire()
		ret = self.__smod.remove(pathbits) 
		self.__lock.release()
		return ret

### === GUI tie-ins	
		
	def getGuiRemoteList(self,rtype,tb):
		tb = self.tuplizePathbits(tb)
		self.__lock.acquire()
		
		if(rtype in ("binder",)):
			if(len(tb) == 0):
				pb = ("binder",)
			else:
				pb = [rtype+"-"+tb[0]]
				pb.extend(tb[1:])			
		else:
			# Either files or "flat" db items
			pb = [rtype]
			pb.extend(tb)			
		tuplist = self.__hmod.specialGuiList(pb)
		self.__lock.release()
		return tuplist
	def getGuiLocalIssueOwner(self,type,issueid,autoadd = False):
		self.__lock.acquire()		
		owner = self.__hmod.specialIssueOwner(issueid)
		if((owner != None) and autoadd):
			parts = type.split("-")
			success = self.__smod.add(("pubs-"+parts[1],owner))
		self.__lock.release()
		return owner
	def getGuiLocalIssues(self,isstype,pubid):
		self.__lock.acquire()			
		ret = []	
		alliss = self.__hmod.specialPubIssues(pubid) # Grab all issue IDs
		root = self.__root				
		for iid in alliss:
			pb2 = (isstype,iid)
			if not self.__smod.exists(pb2):				
				continue # Skip if not subscribed
			fullpath = root + os.sep.join(pb2)
			if not os.path.isdir(fullpath):
				continue # Skip if not existing
			size = os.path.getsize(fullpath)
			modtime = self.localCheck(pb2)
			item = ("folder",iid,'',size,modtime)
			ret.append(item)								
		self.__lock.release()
		return ret

	def getGuiLocalList(self,pb):
		pb = self.tuplizePathbits(pb)
		self.__lock.acquire()
				
		root = self.__root		
		contents = self.listContents(pb,True) # We do want an os.sep decoration				
		
		# Determine their modification time, size, etc.
		# Note that they might not actually exist on disk, we're not sure
		ret = []
		for item in contents:
			expecting_file = True
			if(item[-1] == os.sep):
				expecting_file = False
				item = item[:-1]
			if(pb[0] == "binder"):				
				pb2 = (item,)
				pb2 = self.tuplizePathbits(pb2)
			else:
				pb2 = list(pb[:])
				pb2.append(item)
				pb2 = self.tuplizePathbits(pb2)

			fullpath = root + os.sep.join(pb2)
			if((expecting_file == False) and os.path.isdir(fullpath)):
				t = "folder"
			elif((expecting_file) and os.path.isfile(fullpath)):
				t = "file"
			else:
				# Does not exist/not valid				
				self.notifyDel(pb2) # Treat it like a monitored deletion
				self.log(self.LOG_WARN,	"Could not find subscribed object: "+os.sep.join(pb2))
				continue # Next item
			
			name = str(pb2[-1])			
			modtime = self.localCheck(pb)
				
			if(modtime == 0):
				self.notifyDel(pb2) # Treat it like a monitored deletion
				self.log(self.LOG_WARN,"Could not find subscribed object: "+os.sep.join(pb2))				
				continue # Next item, there was an error
			size = os.path.getsize(fullpath)
				
			ret.append((t,name,'',size,modtime))
				 
		self.__lock.release()
		return ret


		
### === Helper functions


	def remoteCheck(self,pathbits):
		""" Get the modification time of the remote pathbits item if possible. 
	
		The time is adjusted to fit the local clock. e.g. If the server's clock 
		is 1.5 hours ahead, the time is adjusted back 1.5 hours.
		
		Returns None on error."""
		
		pathbits = self.tuplizePathbits(pathbits)
		self.__lock.acquire()
		servtime = self.__hmod.checkRemote(pathbits)
		ourtime = None;
		if(servtime != None):
			ourtime = servtime - self.__hmod.timeDelta()
		self.__lock.release()
		# print("MTIMEDEBUG:\t"+"Remote:"+str(servtime)+" Diff:"+str( self.__hmod.timeDelta())+" Local:"+str(ourtime))
		return ourtime
	def localCheck(self, pathbits):
		""" Get the modification time of a local file, or 0 if it does not exist. """
		pathbits = self.tuplizePathbits(pathbits)
		self.__lock.acquire()
		filepath = self.__root + os.sep.join(pathbits)
		try:
			modtime = int(os.stat(filepath).st_mtime)
		except OSError, e:
			modtime = 0
		self.__lock.release()
		return modtime

	def getLogText(self):
		""" Retrieve and remove the oldest log entry the coordinator has, or a blank string if nothing."""
		self.__lock.acquire()
		try:
			(level,msg) = self.__logtext.pop(0) # shift
			self.__lock.release()
		except IndexError:
			self.__lock.release()
			return ("");
		
		if(level == self.LOG_INFO):
			return "[INFO]\t"+msg
		elif(level == self.LOG_WARN):
			return "[WARN]\t"+msg
		elif(level == self.LOG_ERROR):
			return "[ERROR]\t"+msg
		else:
			return "[????]\t"+msg	
			
	def __log(self,level,text):
		self.__logtext.append((level,text))		
	def log(self,level,text):
		""" Create a log entry with the given loglevel and text. """
		self.__lock.acquire()		
		self.__log(level,text)	
		self.__lock.release()
		print(self.getLogText()) # For debugging

	def tuplizePathbits(self,pb):
		""" Return a sanitized version of the given pathbits representation. 
		
		For example, a string becomes a one-item tuple, a list becomes a tuple, 
		and the contents of any resulting tuple are strings. """
		
		temp = []
		if(type(pb) == tuple):
			temp = list(pb)			
		elif(type(pb) == str):
			temp = [pb,]
			return pb
		elif(type(pb) == list):
			temp = pb
		else:
			assert False
			
		# Stringify everything for consistency
		for i in range(0,len(temp)):
			temp[i] = str(temp[i])
		return tuple(temp)
###
                                                                                                              
	def forceUpload(self,pathbits,data = None):
		""" Upload the given resource to the server. Will not create files or
		database objects the server. Returns false on error.
		
		If the data argument is omitted, it will try to draw the data from
		the local copy. (Error if unable.) """
		
		self.__lock.acquire()
		pathbits = self.tuplizePathbits(pathbits)
		# If data not given, draw from appropriate local file
		ret = False
		if(data == None):
			try:
				filepath = self.__root + os.sep.join(pathbits)
				handle = open(filepath,"r")
				data = handle.read()
				handle.close()
			except OSError,e:
				self.log(self.LOG_WARN,
				"Tried for force an upload, data not given and local file does not exist: "+os.sep.join(pathbits))				
				self.__lock.release()
				return False
				
		try:
			r = self.__hmod.putRemote(pathbits,{'data':data})
			self.__lock.release()
			return True
		except Exception, e:
			self.log(self.LOG_ERROR,str(e))
			self.__lock.release()
			return False
		
		

			
### =================== EVT Del/Add/Mod functions and helpers =================

	
	def notifyDel(self,pathbits):
		""" Inform the coordinator that the given local resource was deleted.
		Typically this is called by the coordinator's modhandler thread. 
		
		If the local resource file no longer exists, it is unsubscribed. A
		notification message may be queued in the guiadapter."""
		pathbits = self.tuplizePathbits(pathbits)
		self.__lock.acquire()
		self.log(self.LOG_INFO,"EVTDEL\t"+str(pathbits))
		modtime = self.remoteCheck(pathbits)
		if(modtime == None):
			# File does not exist on server
			pass # TODO-X notify user	
			self.unsub(pathbits)
		else:
			self.__ga.addFileMessage(pathbits,guiadapter.FILE_DEL)
		self.__lock.release()	

		
	def notifyAdd(self,pathbits):
		""" Inform the coordinator that the given local resource was added.
		Typically this is called by the coordinator's modhandler thread.
		
		Currently this does nothing, since the system by default ignores new
		local file creation in the root directory since it may be a byproduct
		of a dev environment. """
		pathbits = self.tuplizePathbits(pathbits)
		self.__lock.acquire()
		self.log(self.LOG_INFO,"EVTADD\t"+str(pathbits))
		# For the time being, this does nothing--although it's a good
		# idea to program ignore-flags etc. as if it did.
		
		# Basically, creating new files (and especially new database 
		# objects) is something which we probably want more control 
		# over--to make sure that a new issue has a valid (unique) ID, 
		# or that the user has the right permissions on the server, etc.
		
		# If the user wants a new file or DB object, they create it via
		# the GUI, and this app tells the server to allocate something
		# new, and then "gets" the server copy. 
		self.__lock.release()
	def notifyMod(self,pathbits,timestamp):
		""" Inform the coordinator that the given local resource was modified.
		Typically this is called by the coordinator's modhandler thread. 
		
		The coordinator will attempt to upload the file to the server, and may
		queue guiadapter events, for instance if the server's copy claims to be
		newer still."""
		pathbits = self.tuplizePathbits(pathbits)
		self.__lock.acquire()
		self.log(self.LOG_INFO,"EVTMOD\t"+str(pathbits))
		serv_modtime = self.remoteCheck(pathbits)	
		if(serv_modtime == None):
			# Modified, but server copy gone
			pass # TODO-X NOTIFY USER
			self.unsub(pathbits)
			
		else:
			our_modtime = self.localCheck(pathbits)
			if(our_modtime <= 0):
				assert False, "File was deleted since we got the modification message!"
				self.notifyDel(pathbits)
			# So let's compare times. Even if our copy appears older,
			# include a five-second fudge factor.
			aheadby = our_modtime - serv_modtime		
			if(aheadby > (-self.SYNCMARGIN)):
				result = self.forceUpload(pathbits)
				if not result:
					pass #TODO-X Notify user					
					
				else:
					self.log(self.LOG_INFO,"UPLOADED\t"+str(pathbits))
			else:
				self.__ga.addFileMessage(pathbits,guiadapter.FILE_MOD,{'aheadby':(-aheadby)})
		self.__lock.release()
			
		
	def openFile(self,pathbits):
		""" Attempt to open the given local resource for the user to edit with the 
		default handler, dependent on OS and configuration. """		
		pathbits = self.tuplizePathbits(pathbits)
		filepath = self.__root + os.sep.join(pathbits)		
		return(opener.openFile(filepath))
		
		
	
